{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "toc": {
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "colab": {
      "name": "cnn-basic.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3cKF1nqAWkwW",
        "colab_type": "text"
      },
      "source": [
        "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
        "- Author: Sebastian Raschka\n",
        "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4CZ7TeVvWtsN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q IPython\n",
        "!pip install -q ipykernel\n",
        "!pip install -q torch\n",
        "!pip install -q watermark\n",
        "!pip install -q matplotlib\n",
        "!pip install -q tensorwatch\n",
        "!pip install -q sklearn\n",
        "!pip install -q pandas\n",
        "!pip install -q pydot\n",
        "!pip install -q hiddenlayer\n",
        "!pip install -q graphviz"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XVxQi8DdWkwX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "80e1061e-1996-481d-ebf6-a27b5f6510d6"
      },
      "source": [
        "%load_ext watermark\n",
        "%watermark -a 'Sebastian Raschka' -v -p torch"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sebastian Raschka \n",
            "\n",
            "CPython 3.6.9\n",
            "IPython 5.5.0\n",
            "\n",
            "torch 1.5.1+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uYltBu1ZWkwb",
        "colab_type": "text"
      },
      "source": [
        "- Runs on CPU or GPU (if available)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zPpUpWuqWkwb",
        "colab_type": "text"
      },
      "source": [
        "# Model Zoo -- Convolutional Neural Network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u-eu96KmWkwc",
        "colab_type": "text"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3LenG_0aWkwd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dxqJRx4VWkwi",
        "colab_type": "text"
      },
      "source": [
        "## Settings and Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ijKQBoPSWkwi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "0c190255-6675-4dec-d1ae-ae6c583baa62"
      },
      "source": [
        "##########################\n",
        "### SETTINGS\n",
        "##########################\n",
        "\n",
        "# Device\n",
        "device = torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# Hyperparameters\n",
        "random_seed = 1\n",
        "learning_rate = 0.05\n",
        "num_epochs = 10\n",
        "batch_size = 128\n",
        "\n",
        "# Architecture\n",
        "num_classes = 10\n",
        "\n",
        "\n",
        "##########################\n",
        "### MNIST DATASET\n",
        "##########################\n",
        "\n",
        "# Note transforms.ToTensor() scales input images\n",
        "# to 0-1 range\n",
        "train_dataset = datasets.MNIST(root='data', \n",
        "                               train=True, \n",
        "                               transform=transforms.ToTensor(),\n",
        "                               download=True)\n",
        "\n",
        "test_dataset = datasets.MNIST(root='data', \n",
        "                              train=False, \n",
        "                              transform=transforms.ToTensor())\n",
        "\n",
        "\n",
        "train_loader = DataLoader(dataset=train_dataset, \n",
        "                          batch_size=batch_size, \n",
        "                          shuffle=True)\n",
        "\n",
        "test_loader = DataLoader(dataset=test_dataset, \n",
        "                         batch_size=batch_size, \n",
        "                         shuffle=False)\n",
        "\n",
        "# Checking the dataset\n",
        "for images, labels in train_loader:  \n",
        "    print('Image batch dimensions:', images.shape)\n",
        "    print('Image label dimensions:', labels.shape)\n",
        "    break"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
            "Image label dimensions: torch.Size([128])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "guxHgk9WWkwl",
        "colab_type": "text"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c86KNYi_Wkwl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### MODEL\n",
        "##########################\n",
        "\n",
        "\n",
        "class ConvNet(torch.nn.Module):\n",
        "\n",
        "    def __init__(self, num_classes):\n",
        "        super(ConvNet, self).__init__()\n",
        "        \n",
        "        # calculate same padding:\n",
        "        # (w - k + 2*p)/s + 1 = o\n",
        "        # => p = (s(o-1) - w + k)/2\n",
        "        \n",
        "        # 28x28x1 => 28x28x8\n",
        "        self.conv_1 = torch.nn.Conv2d(in_channels=1,\n",
        "                                      out_channels=8,\n",
        "                                      kernel_size=(3, 3),\n",
        "                                      stride=(1, 1),\n",
        "                                      padding=1) # (1(28-1) - 28 + 3) / 2 = 1\n",
        "        # 28x28x8 => 14x14x8\n",
        "        self.pool_1 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
        "                                         stride=(2, 2),\n",
        "                                         padding=0) # (2(14-1) - 28 + 2) = 0                                       \n",
        "        # 14x14x8 => 14x14x16\n",
        "        self.conv_2 = torch.nn.Conv2d(in_channels=8,\n",
        "                                      out_channels=16,\n",
        "                                      kernel_size=(3, 3),\n",
        "                                      stride=(1, 1),\n",
        "                                      padding=1) # (1(14-1) - 14 + 3) / 2 = 1                 \n",
        "        # 14x14x16 => 7x7x16                             \n",
        "        self.pool_2 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
        "                                         stride=(2, 2),\n",
        "                                         padding=0) # (2(7-1) - 14 + 2) = 0\n",
        "\n",
        "        self.linear_1 = torch.nn.Linear(7*7*16, num_classes)\n",
        "\n",
        "        # optionally initialize weights from Gaussian;\n",
        "        # Guassian weight init is not recommended and only for demonstration purposes\n",
        "        for m in self.modules():\n",
        "            if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):\n",
        "                m.weight.data.normal_(0.0, 0.01)\n",
        "                m.bias.data.zero_()\n",
        "                if m.bias is not None:\n",
        "                    m.bias.detach().zero_()\n",
        "        \n",
        "        \n",
        "    def forward(self, x):\n",
        "        out = self.conv_1(x)\n",
        "        out = F.relu(out)\n",
        "        out = self.pool_1(out)\n",
        "\n",
        "        out = self.conv_2(out)\n",
        "        out = F.relu(out)\n",
        "        out = self.pool_2(out)\n",
        "        \n",
        "        logits = self.linear_1(out.view(-1, 7*7*16))\n",
        "        probas = F.softmax(logits, dim=1)\n",
        "        return logits, probas\n",
        "\n",
        "    \n",
        "torch.manual_seed(random_seed)\n",
        "model = ConvNet(num_classes=num_classes)\n",
        "\n",
        "model = model.to(device)\n",
        "\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4p9-Sda-XHDw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "outputId": "7b55a24b-1929-466e-ca48-690b3f7e73e9"
      },
      "source": [
        "import hiddenlayer as hl\n",
        "hl.build_graph(model, torch.zeros([128, 1, 28, 28]).to(device))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<hiddenlayer.graph.Graph at 0x7f5db567ee10>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"1132pt\" height=\"162pt\"\n viewBox=\"0.00 0.00 1132.00 162.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(72 126)\">\n<title>%3</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-72,36 -72,-126 1060,-126 1060,36 -72,36\"/>\n<!-- /outputs/9 -->\n<g id=\"node1\" class=\"node\">\n<title>/outputs/9</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"248,-36 178,-36 178,0 248,0 248,-36\"/>\n<text text-anchor=\"start\" x=\"186\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool2x2</text>\n</g>\n<!-- 16503094806537407946 -->\n<g id=\"node8\" class=\"node\">\n<title>16503094806537407946</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"426,-36 342,-36 342,0 426,0 426,-36\"/>\n<text text-anchor=\"start\" x=\"350\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; Relu</text>\n</g>\n<!-- /outputs/9&#45;&gt;16503094806537407946 -->\n<g id=\"edge6\" class=\"edge\">\n<title>/outputs/9&#45;&gt;16503094806537407946</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M248.017,-18C272.1037,-18 304.5182,-18 331.6582,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"331.7732,-21.5001 341.7732,-18 331.7732,-14.5001 331.7732,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"295\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x8x14x14</text>\n</g>\n<!-- /outputs/12 -->\n<g id=\"node2\" class=\"node\">\n<title>/outputs/12</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"596,-36 526,-36 526,0 596,0 596,-36\"/>\n<text text-anchor=\"start\" x=\"534\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool2x2</text>\n</g>\n<!-- /outputs/14 -->\n<g id=\"node4\" class=\"node\">\n<title>/outputs/14</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"739,-63 685,-63 685,-27 739,-27 739,-63\"/>\n<text text-anchor=\"start\" x=\"695\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Reshape</text>\n</g>\n<!-- /outputs/12&#45;&gt;/outputs/14 -->\n<g id=\"edge1\" class=\"edge\">\n<title>/outputs/12&#45;&gt;/outputs/14</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M596.022,-24.2622C619.5486,-28.4689 650.5202,-34.0069 674.5391,-38.3017\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"674.2388,-41.8034 684.6988,-40.1183 675.471,-34.9127 674.2388,-41.8034\"/>\n<text text-anchor=\"middle\" x=\"640.5\" y=\"-39\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x16x7x7</text>\n</g>\n<!-- /outputs/13 -->\n<g id=\"node3\" class=\"node\">\n<title>/outputs/13</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"588,-90 534,-90 534,-54 588,-54 588,-90\"/>\n<text text-anchor=\"start\" x=\"543\" y=\"-69\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/13&#45;&gt;/outputs/14 -->\n<g id=\"edge2\" class=\"edge\">\n<title>/outputs/13&#45;&gt;/outputs/14</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M588.345,-67.1105C612.5042,-62.7906 647.9539,-56.4519 674.7098,-51.6678\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"675.6307,-55.0587 684.8585,-49.8531 674.3985,-48.168 675.6307,-55.0587\"/>\n</g>\n<!-- /outputs/15 -->\n<g id=\"node5\" class=\"node\">\n<title>/outputs/15</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"866,-63 812,-63 812,-27 866,-27 866,-63\"/>\n<text text-anchor=\"start\" x=\"826\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear</text>\n</g>\n<!-- /outputs/14&#45;&gt;/outputs/15 -->\n<g id=\"edge3\" class=\"edge\">\n<title>/outputs/14&#45;&gt;/outputs/15</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M739.2447,-45C757.492,-45 781.746,-45 801.7693,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"801.8004,-48.5001 811.8004,-45 801.8003,-41.5001 801.8004,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"775.5\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x784</text>\n</g>\n<!-- /outputs/16 -->\n<g id=\"node6\" class=\"node\">\n<title>/outputs/16</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"988,-63 934,-63 934,-27 988,-27 988,-63\"/>\n<text text-anchor=\"start\" x=\"944\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Softmax</text>\n</g>\n<!-- /outputs/15&#45;&gt;/outputs/16 -->\n<g id=\"edge4\" class=\"edge\">\n<title>/outputs/15&#45;&gt;/outputs/16</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M866.0758,-45C883.0553,-45 905.1767,-45 923.7924,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"924,-48.5001 933.9999,-45 923.9999,-41.5001 924,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"900\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x10</text>\n</g>\n<!-- 8938508495585937243 -->\n<g id=\"node7\" class=\"node\">\n<title>8938508495585937243</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"84,-36 0,-36 0,0 84,0 84,-36\"/>\n<text text-anchor=\"start\" x=\"8\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; Relu</text>\n</g>\n<!-- 8938508495585937243&#45;&gt;/outputs/9 -->\n<g id=\"edge5\" class=\"edge\">\n<title>8938508495585937243&#45;&gt;/outputs/9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M84.2697,-18C109.5651,-18 141.7113,-18 167.5519,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"167.8392,-21.5001 177.8391,-18 167.8391,-14.5001 167.8392,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"131\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x8x28x28</text>\n</g>\n<!-- 16503094806537407946&#45;&gt;/outputs/12 -->\n<g id=\"edge7\" class=\"edge\">\n<title>16503094806537407946&#45;&gt;/outputs/12</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M426.3959,-18C453.3301,-18 488.1688,-18 515.6543,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"515.7966,-21.5001 525.7965,-18 515.7965,-14.5001 515.7966,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"476\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x16x14x14</text>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i3nhrgdDXYRb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 439
        },
        "outputId": "96fb283c-5feb-4363-b642-748888bc5d09"
      },
      "source": [
        "import tensorwatch as tw\n",
        "tw.draw_model(model, [128, 1, 28, 28])"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorwatch.model_graph.hiddenlayer.pytorch_draw_model.DotWrapper at 0x7f5da5f70eb8>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: Net Pages: 1 -->\n<svg width=\"2390pt\" height=\"298pt\"\n viewBox=\"0.00 0.00 2390.00 298.48\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 294.4802)\">\n<title>Net</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-294.4802 2386,-294.4802 2386,4 -4,4\"/>\n<!-- top_level_op -->\n<g id=\"node1\" class=\"node\">\n<title>top_level_op</title>\n<path fill=\"#6495ed\" stroke=\"#000000\" d=\"M12,-251.9802C12,-251.9802 90,-251.9802 90,-251.9802 96,-251.9802 102,-257.9802 102,-263.9802 102,-263.9802 102,-277.9802 102,-277.9802 102,-283.9802 96,-289.9802 90,-289.9802 90,-289.9802 12,-289.9802 12,-289.9802 6,-289.9802 0,-283.9802 0,-277.9802 0,-277.9802 0,-263.9802 0,-263.9802 0,-257.9802 6,-251.9802 12,-251.9802\"/>\n<text text-anchor=\"middle\" x=\"51\" y=\"-274.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op</text>\n<text text-anchor=\"middle\" x=\"51\" y=\"-259.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">prim::Constant</text>\n</g>\n<!-- 99 -->\n<g id=\"node6\" class=\"node\">\n<title>99</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"51\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"51\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"51\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- top_level_op&#45;&gt;99 -->\n<g id=\"edge1\" class=\"edge\">\n<title>top_level_op&#45;&gt;99</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M51,-251.8758C51,-244.072 51,-234.7666 51,-225.7253\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"54.5001,-225.7238 51,-215.7238 47.5001,-225.7239 54.5001,-225.7238\"/>\n</g>\n<!-- top_level_op_prim -->\n<g id=\"node2\" class=\"node\">\n<title>top_level_op_prim</title>\n<path fill=\"#6495ed\" stroke=\"#000000\" d=\"M1354,-251.9802C1354,-251.9802 1538,-251.9802 1538,-251.9802 1544,-251.9802 1550,-257.9802 1550,-263.9802 1550,-263.9802 1550,-277.9802 1550,-277.9802 1550,-283.9802 1544,-289.9802 1538,-289.9802 1538,-289.9802 1354,-289.9802 1354,-289.9802 1348,-289.9802 1342,-283.9802 1342,-277.9802 1342,-277.9802 1342,-263.9802 1342,-263.9802 1342,-257.9802 1348,-251.9802 1354,-251.9802\"/>\n<text text-anchor=\"middle\" x=\"1446\" y=\"-274.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op_prim::Constant_41</text>\n<text text-anchor=\"middle\" x=\"1446\" y=\"-259.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">prim::Constant</text>\n</g>\n<!-- top_level_op_aten -->\n<g id=\"node3\" class=\"node\">\n<title>top_level_op_aten</title>\n<path fill=\"#6495ed\" stroke=\"#000000\" d=\"M1580.5,-251.9802C1580.5,-251.9802 1755.5,-251.9802 1755.5,-251.9802 1761.5,-251.9802 1767.5,-257.9802 1767.5,-263.9802 1767.5,-263.9802 1767.5,-277.9802 1767.5,-277.9802 1767.5,-283.9802 1761.5,-289.9802 1755.5,-289.9802 1755.5,-289.9802 1580.5,-289.9802 1580.5,-289.9802 1574.5,-289.9802 1568.5,-283.9802 1568.5,-277.9802 1568.5,-277.9802 1568.5,-263.9802 1568.5,-263.9802 1568.5,-257.9802 1574.5,-251.9802 1580.5,-251.9802\"/>\n<text text-anchor=\"middle\" x=\"1668\" y=\"-274.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op_aten::softmax_42</text>\n<text text-anchor=\"middle\" x=\"1668\" y=\"-259.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">aten::softmax</text>\n</g>\n<!-- logits -->\n<g id=\"node4\" class=\"node\">\n<title>logits</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"540\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"540\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1280</text>\n<text text-anchor=\"middle\" x=\"540\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 10)</text>\n</g>\n<!-- top_level_op_aten: -->\n<g id=\"node48\" class=\"node\">\n<title>top_level_op_aten:</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1203\" cy=\"-107.7401\" rx=\"82.5854\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1203\" y=\"-104.0401\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op_aten:</text>\n</g>\n<!-- logits&#45;&gt;top_level_op_aten: -->\n<g id=\"edge82\" class=\"edge\">\n<title>logits&#45;&gt;top_level_op_aten::softmax_42</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M580.0059,-43.2624C589.3845,-47.3274 599.3918,-51.2336 609,-53.7401 702.1117,-78.0302 970.5153,-95.1989 1112.5253,-103.0069\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1112.3516,-106.5025 1122.5275,-103.5529 1112.7331,-99.5129 1112.3516,-106.5025\"/>\n</g>\n<!-- 97 -->\n<g id=\"node5\" class=\"node\">\n<title>97</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1942\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1942\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1280</text>\n<text text-anchor=\"middle\" x=\"1942\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 10)</text>\n</g>\n<!-- 99&#45;&gt;top_level_op_aten: -->\n<g id=\"edge10\" class=\"edge\">\n<title>99&#45;&gt;top_level_op_aten::_convolution_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M71.0916,-170.4097C75.9825,-166.9254 81.4216,-163.7434 87,-161.7401 181.9848,-127.6301 860.0847,-113.2916 1110.4384,-109.121\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1110.659,-112.618 1120.5999,-108.9534 1110.5435,-105.6189 1110.659,-112.618\"/>\n</g>\n<!-- 100 -->\n<g id=\"node7\" class=\"node\">\n<title>100</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"123\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"123\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"123\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 100&#45;&gt;top_level_op_aten: -->\n<g id=\"edge11\" class=\"edge\">\n<title>100&#45;&gt;top_level_op_aten::_convolution_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M143.0966,-170.4235C147.9872,-166.9385 153.4249,-163.7527 159,-161.7401 247.0906,-129.9402 871.8361,-114.3146 1110.5696,-109.4542\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1110.6878,-112.9526 1120.6151,-109.2514 1110.5465,-105.954 1110.6878,-112.9526\"/>\n</g>\n<!-- 101 -->\n<g id=\"node8\" class=\"node\">\n<title>101</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"195\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"195\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"195\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 101&#45;&gt;top_level_op_aten: -->\n<g id=\"edge12\" class=\"edge\">\n<title>101&#45;&gt;top_level_op_aten::_convolution_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M215.1023,-170.4393C219.9926,-166.9534 225.4288,-163.7633 231,-161.7401 312.2098,-132.2486 883.8366,-115.4175 1110.6176,-109.8376\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1110.9151,-113.3314 1120.8266,-109.5883 1110.7441,-106.3335 1110.9151,-113.3314\"/>\n</g>\n<!-- 116 -->\n<g id=\"node9\" class=\"node\">\n<title>116</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"267\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"267\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"267\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 116&#45;&gt;top_level_op_aten: -->\n<g id=\"edge13\" class=\"edge\">\n<title>116&#45;&gt;top_level_op_aten::_convolution_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M287.109,-170.4576C291.9989,-166.9708 297.4333,-163.7757 303,-161.7401 377.3551,-134.5514 896.2759,-116.6108 1110.6616,-110.2816\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1111.0533,-113.7717 1120.9463,-109.98 1110.848,-106.7747 1111.0533,-113.7717\"/>\n</g>\n<!-- 102 -->\n<g id=\"node10\" class=\"node\">\n<title>102</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"339\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"339\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"339\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 102&#45;&gt;top_level_op_aten: -->\n<g id=\"edge14\" class=\"edge\">\n<title>102&#45;&gt;top_level_op_aten::_convolution_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M359.1169,-170.479C364.0064,-166.9911 369.4386,-163.7901 375,-161.7401 442.5486,-136.8411 909.4682,-117.9014 1110.8687,-110.7961\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1111.2585,-114.2847 1121.1297,-110.4363 1111.0131,-107.289 1111.2585,-114.2847\"/>\n</g>\n<!-- 117 -->\n<g id=\"node11\" class=\"node\">\n<title>117</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"411\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"411\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"411\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 117&#45;&gt;top_level_op_aten: -->\n<g id=\"edge15\" class=\"edge\">\n<title>117&#45;&gt;top_level_op_aten::_convolution_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M431.1263,-170.5045C436.0153,-167.0153 441.4449,-163.8073 447,-161.7401 507.8121,-139.1104 923.719,-119.2952 1111.4238,-111.3917\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1111.7043,-114.8831 1121.549,-110.9676 1111.4113,-107.8892 1111.7043,-114.8831\"/>\n</g>\n<!-- 118 -->\n<g id=\"node12\" class=\"node\">\n<title>118</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"483\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"483\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"483\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 118&#45;&gt;top_level_op_aten: -->\n<g id=\"edge16\" class=\"edge\">\n<title>118&#45;&gt;top_level_op_aten::_convolution_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M503.1378,-170.5353C508.0263,-167.0445 513.4527,-163.828 519,-161.7401 624.68,-121.9636 948.9734,-111.4857 1110.1202,-108.7263\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1110.5537,-112.2197 1120.4946,-108.5554 1110.4383,-105.2206 1110.5537,-112.2197\"/>\n</g>\n<!-- 119 -->\n<g id=\"node13\" class=\"node\">\n<title>119</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"555\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"555\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"555\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 119&#45;&gt;top_level_op_aten: -->\n<g id=\"edge17\" class=\"edge\">\n<title>119&#45;&gt;top_level_op_aten::_convolution_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M575.1522,-170.5732C580.0399,-167.0804 585.4624,-163.8536 591,-161.7401 683.4502,-126.4555 963.3844,-113.9514 1110.4622,-109.7414\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1110.9194,-113.2301 1120.8178,-109.4524 1110.724,-106.2329 1110.9194,-113.2301\"/>\n</g>\n<!-- 120 -->\n<g id=\"node14\" class=\"node\">\n<title>120</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"627\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"627\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"627\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 120&#45;&gt;top_level_op_aten: -->\n<g id=\"edge18\" class=\"edge\">\n<title>120&#45;&gt;top_level_op_aten::_convolution_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M647.1706,-170.621C652.0573,-167.1258 657.4748,-163.8859 663,-161.7401 742.3913,-130.9081 978.965,-116.6923 1111.118,-111.0005\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1111.3862,-114.4924 1121.2294,-110.5729 1111.0904,-107.4987 1111.3862,-114.4924\"/>\n</g>\n<!-- out.1 -->\n<g id=\"node15\" class=\"node\">\n<title>out.1</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"685\" cy=\"-26.8701\" rx=\"76.2353\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"685\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">802816</text>\n<text text-anchor=\"middle\" x=\"685\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 8, 28, 28)</text>\n</g>\n<!-- out.1&#45;&gt;top_level_op_aten: -->\n<g id=\"edge20\" class=\"edge\">\n<title>out.1&#45;&gt;top_level_op_aten::relu_10</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M743.8298,-44.165C755.7588,-47.7388 768.2291,-51.187 780,-53.7401 894.5726,-78.5912 1028.8977,-93.1518 1114.9773,-100.7688\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1114.7044,-104.2582 1124.9709,-101.6412 1115.3131,-97.2848 1114.7044,-104.2582\"/>\n</g>\n<!-- 39 -->\n<g id=\"node16\" class=\"node\">\n<title>39</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"856\" cy=\"-26.8701\" rx=\"76.2353\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"856\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">802816</text>\n<text text-anchor=\"middle\" x=\"856\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 8, 28, 28)</text>\n</g>\n<!-- 39&#45;&gt;top_level_op_aten: -->\n<g id=\"edge27\" class=\"edge\">\n<title>39&#45;&gt;top_level_op_aten::max_pool2d_16</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M915.9225,-43.7615C927.559,-47.2745 939.643,-50.7987 951,-53.7401 1011.7871,-69.4836 1081.4688,-83.7099 1132.1937,-93.6037\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1131.5937,-97.0525 1142.0777,-95.5238 1132.9287,-90.181 1131.5937,-97.0525\"/>\n</g>\n<!-- 103 -->\n<g id=\"node17\" class=\"node\">\n<title>103</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"699\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"699\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"699\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 103&#45;&gt;top_level_op_aten: -->\n<g id=\"edge28\" class=\"edge\">\n<title>103&#45;&gt;top_level_op_aten::max_pool2d_16</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M719.195,-170.6832C724.0805,-167.1847 729.4913,-163.9278 735,-161.7401 801.5007,-135.3305 995.7182,-119.7961 1112.0055,-112.6079\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1112.5472,-116.0815 1122.3159,-111.9798 1112.1215,-109.0945 1112.5472,-116.0815\"/>\n</g>\n<!-- 104 -->\n<g id=\"node18\" class=\"node\">\n<title>104</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"771\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"771\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"771\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 104&#45;&gt;top_level_op_aten: -->\n<g id=\"edge29\" class=\"edge\">\n<title>104&#45;&gt;top_level_op_aten::max_pool2d_16</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M791.229,-170.7674C796.1127,-167.2646 801.5142,-163.9846 807,-161.7401 861.0933,-139.6086 1015.4799,-123.2204 1114.7361,-114.6006\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1115.0971,-118.0826 1124.7611,-113.74 1114.4984,-111.1082 1115.0971,-118.0826\"/>\n</g>\n<!-- 105 -->\n<g id=\"node19\" class=\"node\">\n<title>105</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"843\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"843\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"843\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 105&#45;&gt;top_level_op_aten: -->\n<g id=\"edge30\" class=\"edge\">\n<title>105&#45;&gt;top_level_op_aten::max_pool2d_16</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M863.6656,-170.611C868.4439,-167.2048 873.6915,-164.0044 879,-161.7401 921.1282,-143.7711 1037.9498,-127.0916 1119.1443,-117.1579\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1119.8516,-120.5979 1129.3582,-115.9208 1119.0099,-113.6487 1119.8516,-120.5979\"/>\n</g>\n<!-- 106 -->\n<g id=\"node20\" class=\"node\">\n<title>106</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"915\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"915\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"915\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 106&#45;&gt;top_level_op_aten: -->\n<g id=\"edge31\" class=\"edge\">\n<title>106&#45;&gt;top_level_op_aten::max_pool2d_16</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M935.7466,-170.7949C940.5198,-167.3773 945.7449,-164.1257 951,-161.7401 1003.6951,-137.8188 1067.469,-124.2851 1117.6761,-116.7488\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1118.4566,-120.1724 1127.8517,-115.2752 1117.4533,-113.2447 1118.4566,-120.1724\"/>\n</g>\n<!-- 121 -->\n<g id=\"node21\" class=\"node\">\n<title>121</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"987\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"987\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"987\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 121&#45;&gt;top_level_op_aten: -->\n<g id=\"edge32\" class=\"edge\">\n<title>121&#45;&gt;top_level_op_aten::max_pool2d_16</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1007.8983,-171.1121C1012.6621,-167.6747 1017.845,-164.3349 1023,-161.7401 1057.614,-144.3172 1098.8562,-131.4373 1133.1249,-122.6353\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1134.2065,-125.9726 1143.0562,-120.1475 1132.5054,-119.1824 1134.2065,-125.9726\"/>\n</g>\n<!-- input.2 -->\n<g id=\"node22\" class=\"node\">\n<title>input.2</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1027\" cy=\"-26.8701\" rx=\"76.2353\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1027\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">200704</text>\n<text text-anchor=\"middle\" x=\"1027\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 8, 14, 14)</text>\n</g>\n<!-- input.2&#45;&gt;top_level_op_aten: -->\n<g id=\"edge43\" class=\"edge\">\n<title>input.2&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1077.1597,-47.1865C1104.4227,-59.2303 1137.6109,-74.4699 1163.1059,-86.6474\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1161.5963,-89.8052 1172.1254,-90.9838 1164.6294,-83.4964 1161.5963,-89.8052\"/>\n</g>\n<!-- 107 -->\n<g id=\"node23\" class=\"node\">\n<title>107</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1059\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1059\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1059\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 107&#45;&gt;top_level_op_aten: -->\n<g id=\"edge44\" class=\"edge\">\n<title>107&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1080.2656,-171.7697C1085.0065,-168.2914 1090.0872,-164.7686 1095,-161.7401 1114.8728,-149.4898 1137.8933,-137.6788 1157.5622,-128.2431\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1159.0913,-131.3917 1166.6292,-123.9465 1156.0936,-125.066 1159.0913,-131.3917\"/>\n</g>\n<!-- 108 -->\n<g id=\"node24\" class=\"node\">\n<title>108</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1131\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1131\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1131\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 108&#45;&gt;top_level_op_aten: -->\n<g id=\"edge45\" class=\"edge\">\n<title>108&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1149.1686,-168.2032C1158.6461,-157.5582 1170.3293,-144.4357 1180.387,-133.1389\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1183.0077,-135.4589 1187.0432,-125.6627 1177.7795,-130.8042 1183.0077,-135.4589\"/>\n</g>\n<!-- 109 -->\n<g id=\"node25\" class=\"node\">\n<title>109</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1203\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1203\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1203\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 109&#45;&gt;top_level_op_aten: -->\n<g id=\"edge46\" class=\"edge\">\n<title>109&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1203,-161.6816C1203,-153.4111 1203,-144.2888 1203,-135.9358\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1206.5001,-135.8889 1203,-125.8889 1199.5001,-135.889 1206.5001,-135.8889\"/>\n</g>\n<!-- 122 -->\n<g id=\"node26\" class=\"node\">\n<title>122</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1275\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1275\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1275\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 122&#45;&gt;top_level_op_aten: -->\n<g id=\"edge47\" class=\"edge\">\n<title>122&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1256.8314,-168.2032C1247.3539,-157.5582 1235.6707,-144.4357 1225.613,-133.1389\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1228.2205,-130.8042 1218.9568,-125.6627 1222.9923,-135.4589 1228.2205,-130.8042\"/>\n</g>\n<!-- 110 -->\n<g id=\"node27\" class=\"node\">\n<title>110</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1347\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1347\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1347\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 110&#45;&gt;top_level_op_aten: -->\n<g id=\"edge48\" class=\"edge\">\n<title>110&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1325.7344,-171.7697C1320.9935,-168.2914 1315.9128,-164.7686 1311,-161.7401 1291.1272,-149.4898 1268.1067,-137.6788 1248.4378,-128.2431\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1249.9064,-125.066 1239.3708,-123.9465 1246.9087,-131.3917 1249.9064,-125.066\"/>\n</g>\n<!-- 123 -->\n<g id=\"node28\" class=\"node\">\n<title>123</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1419\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1419\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1419\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 123&#45;&gt;top_level_op_aten: -->\n<g id=\"edge49\" class=\"edge\">\n<title>123&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1398.1017,-171.1121C1393.3379,-167.6747 1388.155,-164.3349 1383,-161.7401 1348.386,-144.3172 1307.1438,-131.4373 1272.8751,-122.6353\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1273.4946,-119.1824 1262.9438,-120.1475 1271.7935,-125.9726 1273.4946,-119.1824\"/>\n</g>\n<!-- 124 -->\n<g id=\"node29\" class=\"node\">\n<title>124</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1491\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1491\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1491\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 124&#45;&gt;top_level_op_aten: -->\n<g id=\"edge50\" class=\"edge\">\n<title>124&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1470.2534,-170.7949C1465.4802,-167.3773 1460.2551,-164.1257 1455,-161.7401 1402.3049,-137.8188 1338.531,-124.2851 1288.3239,-116.7488\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1288.5467,-113.2447 1278.1483,-115.2752 1287.5434,-120.1724 1288.5467,-113.2447\"/>\n</g>\n<!-- 125 -->\n<g id=\"node30\" class=\"node\">\n<title>125</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1563\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1563\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1563\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 125&#45;&gt;top_level_op_aten: -->\n<g id=\"edge51\" class=\"edge\">\n<title>125&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1542.3344,-170.611C1537.5561,-167.2048 1532.3085,-164.0044 1527,-161.7401 1484.8718,-143.7711 1368.0502,-127.0916 1286.8557,-117.1579\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1286.9901,-113.6487 1276.6418,-115.9208 1286.1484,-120.5979 1286.9901,-113.6487\"/>\n</g>\n<!-- 126 -->\n<g id=\"node31\" class=\"node\">\n<title>126</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1635\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1635\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1635\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 126&#45;&gt;top_level_op_aten: -->\n<g id=\"edge52\" class=\"edge\">\n<title>126&#45;&gt;top_level_op_aten::_convolution_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1614.771,-170.7674C1609.8873,-167.2646 1604.4858,-163.9846 1599,-161.7401 1544.9067,-139.6086 1390.5201,-123.2204 1291.2639,-114.6006\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1291.5016,-111.1082 1281.2389,-113.74 1290.9029,-118.0826 1291.5016,-111.1082\"/>\n</g>\n<!-- out -->\n<g id=\"node32\" class=\"node\">\n<title>out</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1203\" cy=\"-26.8701\" rx=\"82.0488\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1203\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">401408</text>\n<text text-anchor=\"middle\" x=\"1203\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 16, 14, 14)</text>\n</g>\n<!-- out&#45;&gt;top_level_op_aten: -->\n<g id=\"edge54\" class=\"edge\">\n<title>out&#45;&gt;top_level_op_aten::relu_27</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1209.4004,-53.7407C1209.8399,-61.9682 1209.865,-71.0431 1209.4756,-79.3673\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1205.9812,-79.1657 1208.7608,-89.3894 1212.9635,-79.6638 1205.9812,-79.1657\"/>\n</g>\n<!-- 72 -->\n<g id=\"node33\" class=\"node\">\n<title>72</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1385\" cy=\"-26.8701\" rx=\"82.0488\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1385\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">401408</text>\n<text text-anchor=\"middle\" x=\"1385\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 16, 14, 14)</text>\n</g>\n<!-- 72&#45;&gt;top_level_op_aten: -->\n<g id=\"edge61\" class=\"edge\">\n<title>72&#45;&gt;top_level_op_aten::max_pool2d_33</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1340.2005,-49.5006C1313.874,-61.5974 1280.5927,-76.3506 1253.3625,-87.9813\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1251.8407,-84.825 1244.0045,-91.9556 1254.577,-91.2681 1251.8407,-84.825\"/>\n</g>\n<!-- 111 -->\n<g id=\"node34\" class=\"node\">\n<title>111</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1707\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1707\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1707\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 111&#45;&gt;top_level_op_aten: -->\n<g id=\"edge62\" class=\"edge\">\n<title>111&#45;&gt;top_level_op_aten::max_pool2d_33</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1686.805,-170.6832C1681.9195,-167.1847 1676.5087,-163.9278 1671,-161.7401 1604.4993,-135.3305 1410.2818,-119.7961 1293.9945,-112.6079\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1293.8785,-109.0945 1283.6841,-111.9798 1293.4528,-116.0815 1293.8785,-109.0945\"/>\n</g>\n<!-- 112 -->\n<g id=\"node35\" class=\"node\">\n<title>112</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1779\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1779\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1779\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 112&#45;&gt;top_level_op_aten: -->\n<g id=\"edge63\" class=\"edge\">\n<title>112&#45;&gt;top_level_op_aten::max_pool2d_33</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1758.8294,-170.621C1753.9427,-167.1258 1748.5252,-163.8859 1743,-161.7401 1663.6087,-130.9081 1427.035,-116.6923 1294.882,-111.0005\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1294.9096,-107.4987 1284.7706,-110.5729 1294.6138,-114.4924 1294.9096,-107.4987\"/>\n</g>\n<!-- 113 -->\n<g id=\"node36\" class=\"node\">\n<title>113</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1851\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1851\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1851\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 113&#45;&gt;top_level_op_aten: -->\n<g id=\"edge64\" class=\"edge\">\n<title>113&#45;&gt;top_level_op_aten::max_pool2d_33</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1830.8478,-170.5732C1825.9601,-167.0804 1820.5376,-163.8536 1815,-161.7401 1722.5498,-126.4555 1442.6156,-113.9514 1295.5378,-109.7414\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1295.276,-106.2329 1285.1822,-109.4524 1295.0806,-113.2301 1295.276,-106.2329\"/>\n</g>\n<!-- 114 -->\n<g id=\"node37\" class=\"node\">\n<title>114</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1923\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1923\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1923\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 114&#45;&gt;top_level_op_aten: -->\n<g id=\"edge65\" class=\"edge\">\n<title>114&#45;&gt;top_level_op_aten::max_pool2d_33</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1902.8622,-170.5353C1897.9737,-167.0445 1892.5473,-163.828 1887,-161.7401 1781.32,-121.9636 1457.0266,-111.4857 1295.8798,-108.7263\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1295.5617,-105.2206 1285.5054,-108.5554 1295.4463,-112.2197 1295.5617,-105.2206\"/>\n</g>\n<!-- 127 -->\n<g id=\"node38\" class=\"node\">\n<title>127</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1995\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1995\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1995\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 127&#45;&gt;top_level_op_aten: -->\n<g id=\"edge66\" class=\"edge\">\n<title>127&#45;&gt;top_level_op_aten::max_pool2d_33</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1974.8737,-170.5045C1969.9847,-167.0153 1964.5551,-163.8073 1959,-161.7401 1898.1879,-139.1104 1482.281,-119.2952 1294.5762,-111.3917\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1294.5887,-107.8892 1284.451,-110.9676 1294.2957,-114.8831 1294.5887,-107.8892\"/>\n</g>\n<!-- 86 -->\n<g id=\"node39\" class=\"node\">\n<title>86</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1556\" cy=\"-26.8701\" rx=\"71.3357\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1556\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">100352</text>\n<text text-anchor=\"middle\" x=\"1556\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 16, 7, 7)</text>\n</g>\n<!-- 86&#45;&gt;top_level_op_aten: -->\n<g id=\"edge69\" class=\"edge\">\n<title>86&#45;&gt;top_level_op_aten::view_35</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1507.9445,-47.0009C1500.3313,-49.4265 1492.5049,-51.7464 1485,-53.7401 1417.4938,-71.6736 1339.1687,-86.5735 1282.152,-96.1468\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1281.4107,-92.7219 1272.1193,-97.8131 1282.5577,-99.6273 1281.4107,-92.7219\"/>\n</g>\n<!-- 115 -->\n<g id=\"node40\" class=\"node\">\n<title>115</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"2067\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"2067\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"2067\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 115&#45;&gt;top_level_op_aten: -->\n<g id=\"edge70\" class=\"edge\">\n<title>115&#45;&gt;top_level_op_aten::view_35</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2046.8831,-170.479C2041.9936,-166.9911 2036.5614,-163.7901 2031,-161.7401 1963.4514,-136.8411 1496.5318,-117.9014 1295.1313,-110.7961\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1294.9869,-107.289 1284.8703,-110.4363 1294.7415,-114.2847 1294.9869,-107.289\"/>\n</g>\n<!-- input -->\n<g id=\"node41\" class=\"node\">\n<title>input</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1701\" cy=\"-26.8701\" rx=\"55.3091\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1701\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">100352</text>\n<text text-anchor=\"middle\" x=\"1701\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 784)</text>\n</g>\n<!-- input&#45;&gt;top_level_op_aten: -->\n<g id=\"edge75\" class=\"edge\">\n<title>input&#45;&gt;top_level_op_aten::addmm_39</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1664.9825,-47.2502C1658.4412,-49.7279 1651.6117,-51.9952 1645,-53.7401 1526.6863,-84.9649 1385.2942,-98.2844 1294.7874,-103.8628\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1294.3276,-100.3839 1284.5541,-104.4739 1294.7449,-107.3715 1294.3276,-100.3839\"/>\n</g>\n<!-- 91 -->\n<g id=\"node42\" class=\"node\">\n<title>91</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1824\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1824\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">7840</text>\n<text text-anchor=\"middle\" x=\"1824\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(784, 10)</text>\n</g>\n<!-- 91&#45;&gt;top_level_op_aten: -->\n<g id=\"edge76\" class=\"edge\">\n<title>91&#45;&gt;top_level_op_aten::addmm_39</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1791.9697,-47.5916C1786.1135,-49.986 1779.9798,-52.1393 1774,-53.7401 1686.3034,-77.2171 1431.2654,-94.9922 1293.5663,-103.0248\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1293.2348,-99.5381 1283.4535,-103.6095 1293.6389,-106.5264 1293.2348,-99.5381\"/>\n</g>\n<!-- 128 -->\n<g id=\"node43\" class=\"node\">\n<title>128</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"2139\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"2139\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"2139\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 128&#45;&gt;top_level_op_aten: -->\n<g id=\"edge77\" class=\"edge\">\n<title>128&#45;&gt;top_level_op_aten::addmm_39</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2118.891,-170.4576C2114.0011,-166.9708 2108.5667,-163.7757 2103,-161.7401 2028.6449,-134.5514 1509.7241,-116.6108 1295.3384,-110.2816\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1295.152,-106.7747 1285.0537,-109.98 1294.9467,-113.7717 1295.152,-106.7747\"/>\n</g>\n<!-- 129 -->\n<g id=\"node44\" class=\"node\">\n<title>129</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"2211\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"2211\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"2211\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 129&#45;&gt;top_level_op_aten: -->\n<g id=\"edge78\" class=\"edge\">\n<title>129&#45;&gt;top_level_op_aten::addmm_39</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2190.8977,-170.4393C2186.0074,-166.9534 2180.5712,-163.7633 2175,-161.7401 2093.7902,-132.2486 1522.1634,-115.4175 1295.3824,-109.8376\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1295.2559,-106.3335 1285.1734,-109.5883 1295.0849,-113.3314 1295.2559,-106.3335\"/>\n</g>\n<!-- 130 -->\n<g id=\"node45\" class=\"node\">\n<title>130</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"2283\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"2283\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"2283\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 130&#45;&gt;top_level_op_aten: -->\n<g id=\"edge83\" class=\"edge\">\n<title>130&#45;&gt;top_level_op_aten::softmax_42</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2262.9034,-170.4235C2258.0128,-166.9385 2252.5751,-163.7527 2247,-161.7401 2158.9094,-129.9402 1534.1639,-114.3146 1295.4304,-109.4542\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1295.4535,-105.954 1285.3849,-109.2514 1295.3122,-112.9526 1295.4535,-105.954\"/>\n</g>\n<!-- 96 -->\n<g id=\"node46\" class=\"node\">\n<title>96</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"2355\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"2355\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"2355\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 96&#45;&gt;top_level_op_aten: -->\n<g id=\"edge84\" class=\"edge\">\n<title>96&#45;&gt;top_level_op_aten::softmax_42</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2334.9084,-170.4097C2330.0175,-166.9254 2324.5784,-163.7434 2319,-161.7401 2224.0152,-127.6301 1545.9153,-113.2916 1295.5616,-109.121\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1295.4565,-105.6189 1285.4001,-108.9534 1295.341,-112.618 1295.4565,-105.6189\"/>\n</g>\n<!-- top_level_op_prim: -->\n<g id=\"node47\" class=\"node\">\n<title>top_level_op_prim:</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1239\" cy=\"-270.9802\" rx=\"85.2851\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1239\" y=\"-267.2802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op_prim:</text>\n</g>\n<!-- top_level_op_prim:&#45;&gt;100 -->\n<g id=\"edge2\" class=\"edge\">\n<title>top_level_op_prim::Constant_1&#45;&gt;100</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1153.7746,-269.4469C915.0986,-264.754 251.4412,-248.8332 159,-215.4802 156.5608,-214.6001 154.1478,-213.4954 151.7955,-212.2392\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"153.4304,-209.1335 143.0961,-206.7982 149.7185,-215.0683 153.4304,-209.1335\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;101 -->\n<g id=\"edge3\" class=\"edge\">\n<title>top_level_op_prim::Constant_2&#45;&gt;101</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1154.1941,-269.1178C927.0943,-263.7355 316.5688,-246.5307 231,-215.4802 228.5624,-214.5957 226.1506,-213.4876 223.7992,-212.229\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"225.4347,-209.1236 215.1016,-206.7831 221.7198,-215.0566 225.4347,-209.1236\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;116 -->\n<g id=\"edge4\" class=\"edge\">\n<title>top_level_op_prim::Constant_3&#45;&gt;116</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1154.4508,-268.7363C939.1636,-262.6309 381.6981,-244.2257 303,-215.4802 300.5642,-214.5905 298.1539,-213.4787 295.8035,-212.2174\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"297.4397,-209.1122 287.1079,-206.7657 293.7213,-215.043 297.4397,-209.1122\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;102 -->\n<g id=\"edge5\" class=\"edge\">\n<title>top_level_op_prim::Constant_4&#45;&gt;102</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1154.6954,-268.293C951.6346,-261.432 446.8516,-241.9255 375,-215.4802 372.5664,-214.5845 370.1577,-213.4682 367.8085,-212.2037\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"369.4455,-209.0989 359.1154,-206.7454 365.7231,-215.0272 369.4455,-209.0989\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;117 -->\n<g id=\"edge6\" class=\"edge\">\n<title>top_level_op_prim::Constant_5&#45;&gt;117</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1155.0939,-267.7775C964.8293,-260.1315 512.0517,-239.6377 447,-215.4802 444.569,-214.5775 442.1623,-213.4559 439.8144,-212.1877\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"441.4525,-209.0832 431.1242,-206.7214 437.7253,-215.0085 441.4525,-209.0832\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;118 -->\n<g id=\"edge7\" class=\"edge\">\n<title>top_level_op_prim::Constant_6&#45;&gt;118</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1155.4966,-267.163C978.5569,-258.6962 577.2843,-237.356 519,-215.4802 516.5722,-214.569 514.1678,-213.4412 511.8216,-212.1685\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"513.4609,-209.0645 503.135,-206.6927 509.728,-214.9862 513.4609,-209.0645\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;119 -->\n<g id=\"edge8\" class=\"edge\">\n<title>top_level_op_prim::Constant_7&#45;&gt;119</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1154.0837,-269.4674C1002.4894,-265.8183 691.7262,-253.7753 591,-215.4802 588.5761,-214.5587 586.1746,-213.4232 583.8304,-212.145\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"585.4713,-209.0417 575.1482,-206.6577 581.7315,-214.9589 585.4713,-209.0417\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;120 -->\n<g id=\"edge9\" class=\"edge\">\n<title>top_level_op_prim::Constant_8&#45;&gt;120</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1154.5671,-268.4141C1016.8656,-263.3023 750.5059,-249.2819 663,-215.4802 660.581,-214.5458 658.1831,-213.4009 655.8416,-212.1158\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"657.4845,-209.0133 647.1649,-206.6141 653.736,-214.925 657.4845,-209.0133\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;103 -->\n<g id=\"edge22\" class=\"edge\">\n<title>top_level_op_prim::Constant_11&#45;&gt;103</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1155.6898,-267.1119C1032.8236,-260.5129 809.5269,-244.8532 735,-215.4802 732.5875,-214.5294 730.1943,-213.3722 727.8562,-212.0785\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"729.5017,-208.977 719.1866,-206.5583 725.7421,-214.8816 729.5017,-208.977\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;104 -->\n<g id=\"edge23\" class=\"edge\">\n<title>top_level_op_prim::Constant_12&#45;&gt;104</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1157.7876,-265.4889C1050.77,-257.4075 868.8597,-240.5097 807,-215.4802 804.5962,-214.5076 802.2094,-213.3343 799.876,-212.029\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"801.5251,-208.929 791.2161,-206.4845 797.7507,-214.8242 801.5251,-208.929\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;105 -->\n<g id=\"edge24\" class=\"edge\">\n<title>top_level_op_prim::Constant_13&#45;&gt;105</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1161.5942,-263.478C1071.4769,-253.973 928.6434,-236.2981 879,-215.4802 876.6086,-214.4774 874.2311,-213.2818 871.9042,-211.9604\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"873.5587,-208.8626 863.2583,-206.3821 869.7637,-214.7446 873.5587,-208.8626\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;106 -->\n<g id=\"edge25\" class=\"edge\">\n<title>top_level_op_prim::Constant_14&#45;&gt;106</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1158.1949,-265.1737C1098.8091,-258.6158 1017.2624,-244.7365 951,-215.4802 948.6902,-214.4604 946.3889,-213.2688 944.1308,-211.9651\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"946.005,-209.009 935.7095,-206.5081 942.1983,-214.8835 946.005,-209.009\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;121 -->\n<g id=\"edge26\" class=\"edge\">\n<title>top_level_op_prim::Constant_15&#45;&gt;121</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1169.9847,-260.3449C1126.3617,-251.927 1069.6248,-237.7928 1023,-215.4802 1020.7224,-214.3903 1018.4452,-213.1463 1016.2046,-211.8046\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1018.0939,-208.858 1007.8213,-206.2651 1014.2349,-214.6982 1018.0939,-208.858\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;107 -->\n<g id=\"edge34\" class=\"edge\">\n<title>top_level_op_prim::Constant_17&#45;&gt;107</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1190.8462,-256.1167C1162.1479,-246.366 1125.4905,-232.3171 1095,-215.4802 1092.7897,-214.2597 1090.5627,-212.9181 1088.3585,-211.5057\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1090.2812,-208.5803 1080.0543,-205.8125 1086.323,-214.3538 1090.2812,-208.5803\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;108 -->\n<g id=\"edge35\" class=\"edge\">\n<title>top_level_op_prim::Constant_18&#45;&gt;108</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1216.096,-253.5117C1200.0261,-241.2554 1178.3828,-224.7484 1160.9455,-211.4492\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1162.8399,-208.4923 1152.7661,-205.2108 1158.5949,-214.0582 1162.8399,-208.4923\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;109 -->\n<g id=\"edge36\" class=\"edge\">\n<title>top_level_op_prim::Constant_19&#45;&gt;109</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1231.0104,-252.6996C1227.1407,-243.8454 1222.3658,-232.9203 1217.8615,-222.6142\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1221.0466,-221.1621 1213.8347,-213.4007 1214.6325,-223.9655 1221.0466,-221.1621\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;122 -->\n<g id=\"edge37\" class=\"edge\">\n<title>top_level_op_prim::Constant_20&#45;&gt;122</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1246.9896,-252.6996C1250.8593,-243.8454 1255.6342,-232.9203 1260.1385,-222.6142\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1263.3675,-223.9655 1264.1653,-213.4007 1256.9534,-221.1621 1263.3675,-223.9655\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;110 -->\n<g id=\"edge38\" class=\"edge\">\n<title>top_level_op_prim::Constant_21&#45;&gt;110</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1261.904,-253.5117C1277.9739,-241.2554 1299.6172,-224.7484 1317.0545,-211.4492\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1319.4051,-214.0582 1325.2339,-205.2108 1315.1601,-208.4923 1319.4051,-214.0582\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;123 -->\n<g id=\"edge39\" class=\"edge\">\n<title>top_level_op_prim::Constant_22&#45;&gt;123</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1287.1538,-256.1167C1315.8521,-246.366 1352.5095,-232.3171 1383,-215.4802 1385.2103,-214.2597 1387.4373,-212.9181 1389.6415,-211.5057\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1391.677,-214.3538 1397.9457,-205.8125 1387.7188,-208.5803 1391.677,-214.3538\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;124 -->\n<g id=\"edge40\" class=\"edge\">\n<title>top_level_op_prim::Constant_23&#45;&gt;124</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1299.3989,-258.2152C1353.2705,-246.3312 1427.7149,-228.5377 1455,-215.4802 1457.2776,-214.3903 1459.5548,-213.1463 1461.7954,-211.8046\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1463.7651,-214.6982 1470.1787,-206.2651 1459.9061,-208.858 1463.7651,-214.6982\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;125 -->\n<g id=\"edge41\" class=\"edge\">\n<title>top_level_op_prim::Constant_24&#45;&gt;125</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1299.3889,-258.2568C1310.5562,-255.9596 1322.1245,-253.6171 1333,-251.4802 1419.0489,-234.573 1446.6556,-250.6235 1527,-215.4802 1529.3133,-214.4684 1531.6173,-213.2827 1533.8772,-211.9834\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1535.8054,-214.9046 1542.3026,-206.5358 1532.0047,-209.0263 1535.8054,-214.9046\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;126 -->\n<g id=\"edge42\" class=\"edge\">\n<title>top_level_op_prim::Constant_25&#45;&gt;126</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1298.3011,-258.0216C1309.7857,-255.6981 1321.7469,-253.4065 1333,-251.4802 1450.5897,-231.3517 1488.5613,-260.5998 1599,-215.4802 1601.4005,-214.4995 1603.7848,-213.3202 1606.1166,-212.0106\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1608.246,-214.8028 1614.7727,-206.457 1604.466,-208.9111 1608.246,-214.8028\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;111 -->\n<g id=\"edge56\" class=\"edge\">\n<title>top_level_op_prim::Constant_28&#45;&gt;111</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1297.7561,-257.8648C1309.3975,-255.5285 1321.556,-253.2722 1333,-251.4802 1482.2531,-228.1089 1530.3518,-270.6236 1671,-215.4802 1673.4142,-214.5337 1675.8086,-213.3797 1678.1476,-212.0882\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1680.2596,-214.893 1686.8191,-206.5729 1676.5028,-208.9865 1680.2596,-214.893\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;112 -->\n<g id=\"edge57\" class=\"edge\">\n<title>top_level_op_prim::Constant_29&#45;&gt;112</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1297.2281,-257.7961C1309.0238,-255.4375 1321.375,-253.1904 1333,-251.4802 1513.9755,-224.8568 1572.0868,-280.6692 1743,-215.4802 1745.4229,-214.5561 1747.8237,-213.4187 1750.1673,-212.1391\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1752.2676,-214.9521 1758.8485,-206.6489 1748.526,-209.0359 1752.2676,-214.9521\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;113 -->\n<g id=\"edge58\" class=\"edge\">\n<title>top_level_op_prim::Constant_30&#45;&gt;113</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1296.9601,-257.7268C1308.8327,-255.3591 1321.2815,-253.1262 1333,-251.4802 1439.3653,-236.5401 1714.3954,-253.1035 1815,-215.4802 1817.4289,-214.5719 1819.8341,-213.4462 1822.1809,-212.175\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1824.2729,-214.9938 1830.8687,-206.7025 1820.542,-209.0709 1824.2729,-214.9938\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;114 -->\n<g id=\"edge59\" class=\"edge\">\n<title>top_level_op_prim::Constant_31&#45;&gt;114</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1296.697,-257.688C1308.646,-255.3084 1321.1911,-253.0808 1333,-251.4802 1455.2529,-234.9102 1771.238,-258.1358 1887,-215.4802 1889.4332,-214.5836 1891.8417,-213.4667 1894.1908,-212.2017\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1896.2766,-215.0248 1902.8835,-206.7423 1892.5536,-209.0969 1896.2766,-215.0248\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;127 -->\n<g id=\"edge60\" class=\"edge\">\n<title>top_level_op_prim::Constant_32&#45;&gt;127</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1296.4373,-257.6693C1308.462,-255.276 1321.1026,-253.0477 1333,-251.4802 1471.1471,-233.2794 1828.0743,-263.1704 1959,-215.4802 1961.4366,-214.5927 1963.8475,-213.4824 1966.1983,-212.2223\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1968.2793,-215.0487 1974.8948,-206.773 1964.5624,-209.117 1968.2793,-215.0487\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;115 -->\n<g id=\"edge68\" class=\"edge\">\n<title>top_level_op_prim::Constant_34&#45;&gt;115</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1296.4302,-257.6148C1308.4554,-255.2254 1321.0981,-253.0125 1333,-251.4802 1487.0459,-231.648 1884.9062,-268.2066 2031,-215.4802 2033.4392,-214.5999 2035.852,-213.495 2038.2043,-212.2387\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2040.2814,-215.0677 2046.9036,-206.7975 2036.5694,-209.133 2040.2814,-215.0677\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;128 -->\n<g id=\"edge73\" class=\"edge\">\n<title>top_level_op_prim::Constant_37&#45;&gt;128</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1296.4245,-257.5704C1308.4502,-255.1842 1321.0944,-252.9839 1333,-251.4802 1502.948,-230.0162 1941.7351,-273.2439 2103,-215.4802 2105.4413,-214.6058 2107.8557,-213.5052 2110.2091,-212.252\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2112.2831,-215.0832 2118.9108,-206.8174 2108.575,-209.146 2112.2831,-215.0832\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;129 -->\n<g id=\"edge74\" class=\"edge\">\n<title>top_level_op_prim::Constant_38&#45;&gt;129</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1296.4199,-257.5336C1308.4459,-255.1501 1321.0914,-252.9601 1333,-251.4802 1518.8525,-228.3841 1998.5616,-278.282 2175,-215.4802 2177.443,-214.6107 2179.8587,-213.5137 2182.213,-212.2631\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2184.2844,-215.0961 2190.9167,-206.8339 2180.5796,-209.1568 2184.2844,-215.0961\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;130 -->\n<g id=\"edge80\" class=\"edge\">\n<title>top_level_op_prim::Constant_40&#45;&gt;130</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1296.4161,-257.5025C1308.4423,-255.1213 1321.089,-252.9401 1333,-251.4802 1534.7588,-226.7518 2055.3864,-283.3208 2247,-215.4802 2249.4445,-214.6148 2251.8613,-213.5208 2254.2163,-212.2724\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2256.2855,-215.1069 2262.9216,-206.8478 2252.5835,-209.166 2256.2855,-215.1069\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;96 -->\n<g id=\"edge81\" class=\"edge\">\n<title>top_level_op_prim::Constant_41&#45;&gt;96</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1295.9121,-257.5753C1308.0879,-255.1572 1320.9214,-252.943 1333,-251.4802 1441.8333,-238.2997 2215.6049,-251.9201 2319,-215.4802 2321.4457,-214.6183 2323.8634,-213.5269 2326.2191,-212.2804\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2328.2865,-215.1162 2334.9258,-206.8597 2324.5868,-209.1738 2328.2865,-215.1162\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;logits -->\n<g id=\"edge79\" class=\"edge\">\n<title>top_level_op_aten::addmm_39&#45;&gt;logits</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1122.1635,-104.0923C979.383,-96.8731 688.6982,-79.2267 591,-53.7401 588.0043,-52.9586 584.9699,-52.0411 581.9492,-51.0251\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"583.0058,-47.6841 572.4127,-47.4911 580.5734,-54.2479 583.0058,-47.6841\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;97 -->\n<g id=\"edge85\" class=\"edge\">\n<title>top_level_op_aten::softmax_42&#45;&gt;97</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1284.6676,-104.8021C1439.9102,-98.6327 1772.0061,-82.4721 1883,-53.7401 1887.4095,-52.5987 1891.8993,-51.1466 1896.3338,-49.5107\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1897.9152,-52.6479 1905.9007,-45.6849 1895.316,-46.1483 1897.9152,-52.6479\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;out.1 -->\n<g id=\"edge19\" class=\"edge\">\n<title>top_level_op_aten::_convolution_9&#45;&gt;out.1</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1123.6786,-102.6197C1034.6775,-95.6334 886.649,-80.7768 762,-53.7401 756.6496,-52.5796 751.1547,-51.2342 745.6778,-49.776\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"746.4463,-46.3569 735.8739,-47.0454 744.5681,-53.1003 746.4463,-46.3569\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;39 -->\n<g id=\"edge21\" class=\"edge\">\n<title>top_level_op_aten::relu_10&#45;&gt;39</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1136.1809,-97.1215C1081.1158,-87.2602 1001.4231,-71.4613 933,-53.7401 928.0171,-52.4496 922.8943,-51.0468 917.7689,-49.5821\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"918.5264,-46.1569 907.9456,-46.702 916.5569,-52.8742 918.5264,-46.1569\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;input.2 -->\n<g id=\"edge33\" class=\"edge\">\n<title>top_level_op_aten::max_pool2d_16&#45;&gt;input.2</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1163.1666,-91.9556C1138.252,-81.1004 1105.9795,-66.4123 1079.0702,-53.7366\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1080.3326,-50.4618 1069.7966,-49.3465 1077.3375,-56.7887 1080.3326,-50.4618\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;out -->\n<g id=\"edge53\" class=\"edge\">\n<title>top_level_op_aten::_convolution_26&#45;&gt;out</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1197.2392,-89.3894C1196.4834,-81.8196 1196.1607,-72.7601 1196.2711,-63.9182\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1199.7751,-63.8484 1196.5996,-53.7407 1192.7787,-63.6225 1199.7751,-63.8484\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;72 -->\n<g id=\"edge55\" class=\"edge\">\n<title>top_level_op_aten::relu_27&#45;&gt;72</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1235.1178,-90.9838C1259.4095,-79.6288 1293.3918,-64.4272 1322.8927,-51.6763\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1324.4931,-54.798 1332.2955,-47.6305 1321.7264,-48.3679 1324.4931,-54.798\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;86 -->\n<g id=\"edge67\" class=\"edge\">\n<title>top_level_op_aten::max_pool2d_33&#45;&gt;86</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1266.9144,-96.2322C1320.7025,-86.6621 1399.2929,-71.7269 1467,-53.7401 1474.6509,-51.7076 1482.636,-49.3361 1490.5579,-46.8592\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1491.6958,-50.1701 1500.1628,-43.8014 1489.5723,-43.5 1491.6958,-50.1701\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;input -->\n<g id=\"edge71\" class=\"edge\">\n<title>top_level_op_aten::view_35&#45;&gt;input</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1283.7588,-103.7028C1369.9618,-98.0313 1509.8032,-84.6701 1627,-53.7401 1633.8184,-51.9407 1640.8682,-49.5857 1647.8015,-47.0174\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1649.0781,-50.2764 1657.1511,-43.4151 1646.5614,-43.7444 1649.0781,-50.2764\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;91 -->\n<g id=\"edge72\" class=\"edge\">\n<title>top_level_op_aten::t_36&#45;&gt;91</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1282.8938,-102.9937C1413.9242,-94.9462 1668.4167,-77.1868 1756,-53.7401 1762.1361,-52.0974 1768.4343,-49.8731 1774.6212,-47.4034\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1776.352,-50.4724 1784.2126,-43.3688 1773.6378,-44.02 1776.352,-50.4724\"/>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ArAho4-9Wkwn",
        "colab_type": "text"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7uo6Rj5LWkwo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "37ab213f-074e-4f3e-cce9-bae8c783a4e4"
      },
      "source": [
        "def compute_accuracy(model, data_loader):\n",
        "    correct_pred, num_examples = 0, 0\n",
        "    for features, targets in data_loader:\n",
        "        features = features.to(device)\n",
        "        targets = targets.to(device)\n",
        "        logits, probas = model(features)\n",
        "        _, predicted_labels = torch.max(probas, 1)\n",
        "        num_examples += targets.size(0)\n",
        "        correct_pred += (predicted_labels == targets).sum()\n",
        "    return correct_pred.float()/num_examples * 100\n",
        "    \n",
        "\n",
        "start_time = time.time()    \n",
        "for epoch in range(num_epochs):\n",
        "    model = model.train()\n",
        "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
        "        \n",
        "        features = features.to(device)\n",
        "        targets = targets.to(device)\n",
        "\n",
        "        ### FORWARD AND BACK PROP\n",
        "        logits, probas = model(features)\n",
        "        cost = F.cross_entropy(logits, targets)\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        cost.backward()\n",
        "        \n",
        "        ### UPDATE MODEL PARAMETERS\n",
        "        optimizer.step()\n",
        "        \n",
        "        ### LOGGING\n",
        "        if not batch_idx % 50:\n",
        "            print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
        "                   %(epoch+1, num_epochs, batch_idx, \n",
        "                     len(train_loader), cost))\n",
        "    \n",
        "    model = model.eval()\n",
        "    print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
        "          epoch+1, num_epochs, \n",
        "          compute_accuracy(model, train_loader)))\n",
        "\n",
        "    print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
        "    \n",
        "print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 001/010 | Batch 000/469 | Cost: 2.3026\n",
            "Epoch: 001/010 | Batch 050/469 | Cost: 2.3010\n",
            "Epoch: 001/010 | Batch 100/469 | Cost: 2.3024\n",
            "Epoch: 001/010 | Batch 150/469 | Cost: 2.3041\n",
            "Epoch: 001/010 | Batch 200/469 | Cost: 2.3010\n",
            "Epoch: 001/010 | Batch 250/469 | Cost: 2.3017\n",
            "Epoch: 001/010 | Batch 300/469 | Cost: 2.3071\n",
            "Epoch: 001/010 | Batch 350/469 | Cost: 2.2917\n",
            "Epoch: 001/010 | Batch 400/469 | Cost: 2.2958\n",
            "Epoch: 001/010 | Batch 450/469 | Cost: 2.2072\n",
            "Epoch: 001/010 training accuracy: 58.34%\n",
            "Time elapsed: 0.39 min\n",
            "Epoch: 002/010 | Batch 000/469 | Cost: 1.8305\n",
            "Epoch: 002/010 | Batch 050/469 | Cost: 1.2146\n",
            "Epoch: 002/010 | Batch 100/469 | Cost: 0.4155\n",
            "Epoch: 002/010 | Batch 150/469 | Cost: 0.3560\n",
            "Epoch: 002/010 | Batch 200/469 | Cost: 0.2075\n",
            "Epoch: 002/010 | Batch 250/469 | Cost: 0.3605\n",
            "Epoch: 002/010 | Batch 300/469 | Cost: 0.3065\n",
            "Epoch: 002/010 | Batch 350/469 | Cost: 0.3635\n",
            "Epoch: 002/010 | Batch 400/469 | Cost: 0.2209\n",
            "Epoch: 002/010 | Batch 450/469 | Cost: 0.2445\n",
            "Epoch: 002/010 training accuracy: 92.98%\n",
            "Time elapsed: 0.78 min\n",
            "Epoch: 003/010 | Batch 000/469 | Cost: 0.2873\n",
            "Epoch: 003/010 | Batch 050/469 | Cost: 0.1673\n",
            "Epoch: 003/010 | Batch 100/469 | Cost: 0.1826\n",
            "Epoch: 003/010 | Batch 150/469 | Cost: 0.1675\n",
            "Epoch: 003/010 | Batch 200/469 | Cost: 0.3072\n",
            "Epoch: 003/010 | Batch 250/469 | Cost: 0.1825\n",
            "Epoch: 003/010 | Batch 300/469 | Cost: 0.1812\n",
            "Epoch: 003/010 | Batch 350/469 | Cost: 0.1564\n",
            "Epoch: 003/010 | Batch 400/469 | Cost: 0.1302\n",
            "Epoch: 003/010 | Batch 450/469 | Cost: 0.1391\n",
            "Epoch: 003/010 training accuracy: 95.85%\n",
            "Time elapsed: 1.17 min\n",
            "Epoch: 004/010 | Batch 000/469 | Cost: 0.1122\n",
            "Epoch: 004/010 | Batch 050/469 | Cost: 0.1861\n",
            "Epoch: 004/010 | Batch 100/469 | Cost: 0.1325\n",
            "Epoch: 004/010 | Batch 150/469 | Cost: 0.0864\n",
            "Epoch: 004/010 | Batch 200/469 | Cost: 0.1003\n",
            "Epoch: 004/010 | Batch 250/469 | Cost: 0.1865\n",
            "Epoch: 004/010 | Batch 300/469 | Cost: 0.1974\n",
            "Epoch: 004/010 | Batch 350/469 | Cost: 0.1975\n",
            "Epoch: 004/010 | Batch 400/469 | Cost: 0.1290\n",
            "Epoch: 004/010 | Batch 450/469 | Cost: 0.1305\n",
            "Epoch: 004/010 training accuracy: 96.81%\n",
            "Time elapsed: 1.57 min\n",
            "Epoch: 005/010 | Batch 000/469 | Cost: 0.0755\n",
            "Epoch: 005/010 | Batch 050/469 | Cost: 0.0977\n",
            "Epoch: 005/010 | Batch 100/469 | Cost: 0.1275\n",
            "Epoch: 005/010 | Batch 150/469 | Cost: 0.1049\n",
            "Epoch: 005/010 | Batch 200/469 | Cost: 0.1097\n",
            "Epoch: 005/010 | Batch 250/469 | Cost: 0.1460\n",
            "Epoch: 005/010 | Batch 300/469 | Cost: 0.0882\n",
            "Epoch: 005/010 | Batch 350/469 | Cost: 0.1439\n",
            "Epoch: 005/010 | Batch 400/469 | Cost: 0.2756\n",
            "Epoch: 005/010 | Batch 450/469 | Cost: 0.1580\n",
            "Epoch: 005/010 training accuracy: 96.93%\n",
            "Time elapsed: 1.96 min\n",
            "Epoch: 006/010 | Batch 000/469 | Cost: 0.1547\n",
            "Epoch: 006/010 | Batch 050/469 | Cost: 0.0739\n",
            "Epoch: 006/010 | Batch 100/469 | Cost: 0.1348\n",
            "Epoch: 006/010 | Batch 150/469 | Cost: 0.0741\n",
            "Epoch: 006/010 | Batch 200/469 | Cost: 0.1078\n",
            "Epoch: 006/010 | Batch 250/469 | Cost: 0.0246\n",
            "Epoch: 006/010 | Batch 300/469 | Cost: 0.1561\n",
            "Epoch: 006/010 | Batch 350/469 | Cost: 0.1545\n",
            "Epoch: 006/010 | Batch 400/469 | Cost: 0.0791\n",
            "Epoch: 006/010 | Batch 450/469 | Cost: 0.1028\n",
            "Epoch: 006/010 training accuracy: 95.55%\n",
            "Time elapsed: 2.35 min\n",
            "Epoch: 007/010 | Batch 000/469 | Cost: 0.1272\n",
            "Epoch: 007/010 | Batch 050/469 | Cost: 0.0819\n",
            "Epoch: 007/010 | Batch 100/469 | Cost: 0.0593\n",
            "Epoch: 007/010 | Batch 150/469 | Cost: 0.0374\n",
            "Epoch: 007/010 | Batch 200/469 | Cost: 0.0626\n",
            "Epoch: 007/010 | Batch 250/469 | Cost: 0.2045\n",
            "Epoch: 007/010 | Batch 300/469 | Cost: 0.1377\n",
            "Epoch: 007/010 | Batch 350/469 | Cost: 0.0196\n",
            "Epoch: 007/010 | Batch 400/469 | Cost: 0.0909\n",
            "Epoch: 007/010 | Batch 450/469 | Cost: 0.0583\n",
            "Epoch: 007/010 training accuracy: 97.36%\n",
            "Time elapsed: 2.75 min\n",
            "Epoch: 008/010 | Batch 000/469 | Cost: 0.0399\n",
            "Epoch: 008/010 | Batch 050/469 | Cost: 0.0483\n",
            "Epoch: 008/010 | Batch 100/469 | Cost: 0.0680\n",
            "Epoch: 008/010 | Batch 150/469 | Cost: 0.1427\n",
            "Epoch: 008/010 | Batch 200/469 | Cost: 0.1115\n",
            "Epoch: 008/010 | Batch 250/469 | Cost: 0.0579\n",
            "Epoch: 008/010 | Batch 300/469 | Cost: 0.0757\n",
            "Epoch: 008/010 | Batch 350/469 | Cost: 0.1057\n",
            "Epoch: 008/010 | Batch 400/469 | Cost: 0.1164\n",
            "Epoch: 008/010 | Batch 450/469 | Cost: 0.0736\n",
            "Epoch: 008/010 training accuracy: 97.42%\n",
            "Time elapsed: 3.14 min\n",
            "Epoch: 009/010 | Batch 000/469 | Cost: 0.0158\n",
            "Epoch: 009/010 | Batch 050/469 | Cost: 0.0698\n",
            "Epoch: 009/010 | Batch 100/469 | Cost: 0.0497\n",
            "Epoch: 009/010 | Batch 150/469 | Cost: 0.0587\n",
            "Epoch: 009/010 | Batch 200/469 | Cost: 0.0288\n",
            "Epoch: 009/010 | Batch 250/469 | Cost: 0.0651\n",
            "Epoch: 009/010 | Batch 300/469 | Cost: 0.0646\n",
            "Epoch: 009/010 | Batch 350/469 | Cost: 0.0464\n",
            "Epoch: 009/010 | Batch 400/469 | Cost: 0.0548\n",
            "Epoch: 009/010 | Batch 450/469 | Cost: 0.0794\n",
            "Epoch: 009/010 training accuracy: 97.77%\n",
            "Time elapsed: 3.53 min\n",
            "Epoch: 010/010 | Batch 000/469 | Cost: 0.1288\n",
            "Epoch: 010/010 | Batch 050/469 | Cost: 0.0507\n",
            "Epoch: 010/010 | Batch 100/469 | Cost: 0.0691\n",
            "Epoch: 010/010 | Batch 150/469 | Cost: 0.0613\n",
            "Epoch: 010/010 | Batch 200/469 | Cost: 0.0621\n",
            "Epoch: 010/010 | Batch 250/469 | Cost: 0.0781\n",
            "Epoch: 010/010 | Batch 300/469 | Cost: 0.0223\n",
            "Epoch: 010/010 | Batch 350/469 | Cost: 0.1057\n",
            "Epoch: 010/010 | Batch 400/469 | Cost: 0.0265\n",
            "Epoch: 010/010 | Batch 450/469 | Cost: 0.0264\n",
            "Epoch: 010/010 training accuracy: 97.79%\n",
            "Time elapsed: 3.92 min\n",
            "Total Training Time: 3.92 min\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DkezE2m3Wkwq",
        "colab_type": "text"
      },
      "source": [
        "## Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WGkF88RCWkwq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "859a2b6b-6172-494e-8085-065e2ac3afa8"
      },
      "source": [
        "with torch.set_grad_enabled(False): # save memory during inference\n",
        "    print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy: 97.84%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BYVWMGiwWkwt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "f53bb3f1-f973-4e92-b188-929f6e744163"
      },
      "source": [
        "%watermark -iv"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "numpy 1.18.5\n",
            "torch 1.5.1+cu101\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}